import torch
import torch.nn as nn


class LayerNorm(nn.Module):

    def __init__(self, feature_dim, eps=1e-6):
        """
        初始化函数有两个参数, 一个是features, 表示词嵌入的维度,
        另一个是eps它是一个足够小的数, 在规范化公式的分母中出现,
        防止分母为0.默认是1e-6.
        """
        super(LayerNorm, self).__init__()
        self.gamma = nn.Parameter(torch.ones(feature_dim))
        self.beta = nn.Parameter(torch.zeros(feature_dim))
        self.eps = eps

    def forward(self, x):
        """输入参数x代表来自上一层的输出"""
        mean = x.mean(-1, keepdim=True)
        std = x.std(-1, keepdim=True)
        return self.gamma * (x - mean) / (std + self.eps) + self.beta